CUDA: enable cuda support v1 - EAGER with GDR COPY #20

bureddy · 2017-09-26T22:42:57Z

    -   configuration
    -   UCT interface changes to detect address domain
    -   GDR COPY UCT
              - put-zcopy func to write data from bounce buffer to domain(CUDA) buffer 
    -   CUDA COPY UCT
              - get-zcopy func to stage data from domain buffer to bounce buffer
   -    UCP protocol changes for CUDA eager zcopy
             - domain detection
             - domain unpack
             - domain uct lanes
   -   mpool for cuda rndv staging
   -  cudaFree memhooks

Pending:
    -  rndv pipeline transfers using PUT protocol

- for pipeline rndv transfers

hoopoepg · 2017-09-29T05:54:00Z

src/ucm/cuda/cudamem.h

@@ -0,0 +1,22 @@
+/**
+ * Copyright (C) Mellanox Technologies Ltd. 2001-2015.  ALL RIGHTS RESERVED.


copyright year 2017?

hoopoepg · 2017-09-29T05:56:48Z

src/ucm/cuda/install.c

+    ucm_debug("cudamem test: got 0x%x out of 0x%x, total: 0x%x", out_events, events,
+              installed_events);
+
+    /* Return success iff we caught all wanted events */


hoopoepg · 2017-09-29T06:05:36Z

src/ucm/cuda/replace.c

+#include <pthread.h>
+
+
+#define MAP_FAILED ((void*)-1)


is it required? MAP_FAILED is defined in <sys/mman.h>, and MAP_FAILED is not used here

hoopoepg · 2017-09-29T06:11:03Z

src/ucp/core/ucp_ep.c

+        iface_attr = &worker->ifaces[rsc_index].attr;
+
+        domain_config->tag.eager.max_short = iface_attr->cap.am.max_short;
+        //TODO: zcopy thrshold should be based on the ep AM lane capability with domain addr(i.e  can UCT  do zcopy from domain) 


thrshold -> threshold

hoopoepg · 2017-09-29T06:14:36Z

src/ucp/core/ucp_ep.h

@@ -106,6 +109,17 @@ typedef struct ucp_ep_rma_config {
 } ucp_ep_rma_config_t;


+#define UCP_IS_DEFAULT_ADDR_DOMAIN(_addr_dn_h) (_addr_dn_h == &ucp_addr_dn_dummy_handle)


(_addr_dn_h == &ucp_addr_dn_dummy_handle) -> ((_addr_dn_h) == &ucp_addr_dn_dummy_handle)

hoopoepg · 2017-09-29T06:26:29Z

src/ucp/dt/dt.c

+        return UCS_OK;
+    }
+
+    while(1) {


space after while:
while (1) {

hoopoepg · 2017-09-29T06:28:50Z

src/ucp/tag/rndv.c

-                               rreq->recv.length, &rreq->recv.state,
-                               data + hdr_len, recv_len, 1);
+        status = ucp_dt_unpack(rreq, rreq->recv.datatype, rreq->recv.buffer,
+                rreq->recv.length, &rreq->recv.state,


hoopoepg · 2017-09-29T06:33:17Z

src/ucp/wireup/select.c

+
+    dn_md_map = addr_dn_h->md_map;
+
+        while(1) {


coding guide line: space after while: while (1) {

hoopoepg · 2017-09-29T06:39:51Z

src/uct/cuda/cuda_copy/cuda_copy_iface.c

+    uct_cuda_copy_event_desc_t *cuda_event;
+    cudaError_t result = cudaSuccess;
+
+    ucs_queue_for_each_safe(cuda_event, iter, &iface->pending_event_q, queue)


coding guideline: ucs_queue_for_each_safe() {

hoopoepg · 2017-09-29T06:42:32Z

src/uct/cuda/cuda_copy/cuda_copy_iface.c

+        ucs_error("cudaEventCreateWithFlags Failed");
+    }
+}
+void uct_cuda_copy_event_desc_cleanup(ucs_mpool_t *mp, void *obj)


you don't call these functions directly by name from anywhere, may be declare it as 'static'?

bureddy · 2017-10-09T15:29:00Z

src/uct/api/uct.h

@@ -630,6 +643,7 @@ struct uct_md_attr {
        size_t               max_alloc; /**< Maximal allocation size */
        size_t               max_reg;   /**< Maximal registration size */
        uint64_t             flags;     /**< UCT_MD_FLAG_xx */
+        uct_addr_domain_t    addr_dn;   /**< Supported addr domain */


change to uct_memory_domain_t

or memory_type , memory_kind

bureddy · 2017-10-09T15:29:10Z

src/uct/api/uct.h

+typedef enum {
+    UCT_MD_ADDR_DOMAIN_CUDA = 0,  /**< NVIDIA CUDA domain */
+    UCT_MD_ADDR_DOMAIN_DEFAULT,   /**< Default system domain */
+    UCT_MD_ADDR_DOMAIN_LAST = UCT_MD_ADDR_DOMAIN_DEFAULT


bureddy · 2017-10-09T15:29:24Z

src/uct/api/uct.h

                                               connection establishment via
                                               sockaddr */
+    UCT_MD_FLAG_ADDR_DN    = UCS_BIT(8)   /**< MD supports memory addr domain


change to _MRM_DETECT

bureddy · 2017-10-09T15:30:20Z

src/uct/api/uct.h

+ * @param [in]     md        Memory domain to register memory on.
+ * @param [in]     address   Memory address to detect.
+ */
+ucs_status_t uct_md_mem_detect(uct_md_h md, void *addr);


add to ucx_info to show memory detect support

bureddy · 2017-10-09T15:37:01Z

src/ucp/core/ucp_mm.c

+    /*TODO: return if no MDs with address domain detect */
+
+    for (md_index = 0; md_index < context->num_mds; ++md_index) {
+        if (context->tl_mds[md_index].attr.cap.flags & UCT_MD_FLAG_ADDR_DN) {


remove cap flag and use UCT_MD_ADDR_DOMAIN_DEFAULT

bureddy · 2017-10-09T15:40:01Z

src/ucp/core/ucp_request.h

@@ -78,6 +78,7 @@ typedef void (*ucp_request_callback_t)(ucp_request_t *req);
 struct ucp_request {
    ucs_status_t                  status;  /* Operation status */
    uint16_t                      flags;   /* Request flags */
+    ucp_addr_dn_h                 addr_dn_h;  /* Memory domain handle */


embeded fields into req struct

bureddy · 2017-10-09T15:47:45Z

src/ucp/tag/tag_send.c

-                           ucp_ep_config(ep)->tag.eager.max_short,
-                           ucp_ep_config(ep)->tag.eager.zcopy_thresh,
+                           ucs_likely(UCP_IS_DEFAULT_ADDR_DOMAIN(addr_dn_h)) ?
+                                    ucp_ep_config(ep)->tag.eager.max_short :


array of eager threshold per domain

bureddy · 2017-10-09T15:48:23Z

src/ucp/tag/tag_send.c


    UCP_THREAD_CS_ENTER_CONDITIONAL(&ep->worker->mt_lock);

    ucs_trace_req("send_nb buffer %p count %zu tag %"PRIx64" to %s cb %p",
                  buffer, count, tag, ucp_ep_peer_name(ep), cb);

-    if (ucs_likely(UCP_DT_IS_CONTIG(datatype))) {
+    ucp_addr_domain_detect_mds(ep->worker->context, (void *)buffer, &addr_dn_h);


global context value to check if detect is needed or not

bureddy force-pushed the cuda-domain branch from 5a4d70e to b3bde76 Compare September 27, 2017 17:41

bureddy added 9 commits September 28, 2017 12:13

CUDA: configuration flags

a1b6607

remove CUDA UCT

87d29ef

CUDA; GDR COPY UCT implementation

ff3bcd0

CUDA: CUDA COPY UCT

4ed2a7d

- for pipeline rndv transfers

CUDA: UCT interface changes for support address domains

87ecdd7

CUDA: cudaFree() memhook

9cbca08

CUDA: UCP address domain detect func

4c1417b

CUDA: protocol changes for eager support for domain memory

369d416

CUDA: mpool for pipeline staging for domain memory

2915cdb

bureddy force-pushed the cuda-domain branch from b3bde76 to 2915cdb Compare September 28, 2017 19:15

bureddy changed the title ~~CUDA: enable cuda support v1~~ CUDA: enable cuda support v1 - EAGER with GDR COPY Sep 28, 2017

hoopoepg reviewed Sep 29, 2017

View reviewed changes

src/ucm/cuda/cudamem.h

@@ -0,0 +1,22 @@

/**

* Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED.

Copy link

hoopoepg Sep 29, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

copyright year 2017?

hoopoepg reviewed Sep 29, 2017

View reviewed changes

bureddy commented Oct 9, 2017

View reviewed changes

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CUDA: enable cuda support v1 - EAGER with GDR COPY #20

CUDA: enable cuda support v1 - EAGER with GDR COPY #20

bureddy commented Sep 26, 2017 •

edited

Loading

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

hoopoepg Sep 29, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017 •

edited

Loading

bureddy Oct 9, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017

bureddy Oct 9, 2017

		@@ -0,0 +1,22 @@
		/**
		* Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED.

		@@ -106,6 +109,17 @@ typedef struct ucp_ep_rma_config {
		} ucp_ep_rma_config_t;


		#define UCP_IS_DEFAULT_ADDR_DOMAIN(_addr_dn_h) (_addr_dn_h == &ucp_addr_dn_dummy_handle)

CUDA: enable cuda support v1 - EAGER with GDR COPY #20

Are you sure you want to change the base?

CUDA: enable cuda support v1 - EAGER with GDR COPY #20

Conversation

bureddy commented Sep 26, 2017 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

bureddy Oct 9, 2017 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

bureddy commented Sep 26, 2017 •

edited

Loading

bureddy Oct 9, 2017 •

edited

Loading